#!/bin/bash
# This is the script for running the NAS Parallel Benchmark
#
###SHELLPACK preamble nas-bench 3.3.1

if [ "$NAS_OPENMPI_VERSION" = "openmpi2" ]; then
	install-depends openmpi2 openmpi2-devel
else
	install-depends openmpi openmpi-devel
fi
install-depends libpsm_infinipath1
install-depends gfortran gcc

NAS_TYPE=SER
RUNBITS=
OPENMP=
OPENMPI=
MPICPUS=
MEMTOTAL_MBYTES=$((MEMTOTAL_BYTES/1048576))
NAS_BIND_PLACE=${NAS_BIND_PLACE:-socket}

if [ "$NAS_MAX_CPUS" = "" ]; then
	NAS_MAX_CPUS=$NUMCPUS
fi

if [ "$NAS_MPI_PATH" != "" ]; then
	export PATH=$PATH:$NAS_MPI_PATH
fi

if [ "$NAS_MPI_LIBPATH" != "" ]; then
	export LD_LIBRARY_PATH=$NAS_MPI_LIBPATH
fi

###SHELLPACK parseargBegin
	--type)
		NAS_TYPE=$2
		MPIOPT="-mca btl ^openib,udapl"
		if [ "$NAS_TYPE" = "OMP" ]; then
			OPENMP=--openmp
		fi
		if [ "$NAS_TYPE" = "MPI" ]; then
			export OMPI_TMPDIR=$SHELLPACK_DATA
			export TMPDIR=$SHELLPACK_DATA
			OPENMPI=--openmpi
			case $NAS_CLASS in
			C)
				if [ $MEMTOTAL_MBYTES -lt 6144 ]; then
					echo INFO: Machine has too little memory to be useful
					exit $SHELLPACK_SUCCESS
				fi
				;;
			*)
				echo Machine has sufficient memory for testing.
				;;
			esac
		fi
		shift 2
		;;
	--bitness)
		RUNBITS=$2
		shift 2
		;;
	--cma)
		MPIOPT="-mca btl btl_sm_use_cma 1"
		shift
		;;
###SHELLPACK parseargParam --max-cpus   NAS_MAX_CPUS
###SHELLPACK parseargParam --iterations NAS_ITERATIONS
###SHELLPACK parseargParam --joblist    NAS_JOBLIST
###SHELLPACK parseargEnd
###SHELLPACK monitor_hooks

###SHELLPACK check_install_required nas-$VERSION
###SHELLPACK init_complete
###SHELLPACK build_start nas-$VERSION-installed

JOBLIST="bt cg dc ep ft is lu mg sp ua"
if [ "$NAS_JOBLIST" != "" ]; then
	JOBLIST=$NAS_JOBLIST
fi

# Additional test parameters
case $NAS_TYPE in
SER)
	echo
	;;
MPI)
	REMOVE_LIST="dc ua"
	if [ "$NAS_CLASS" = "D" ]; then
		REMOVE_LIST="$REMOVE_LIST ft"
	fi
	for REMOVE in $REMOVE_LIST; do
		JOBLIST=`echo $JOBLIST | sed -e "s/$REMOVE//"`
	done
	if [ "$NAS_BIND" = "yes" ]; then
		MMTESTS_MPI_OPTS+=" --bind-to $NAS_BIND_PLACE"
	fi
	MPIRUN="mpirun $NAS_MPI_OPTS $MMTESTS_MPI_OPTS"
	;;
OMP)
	REMOVE_LIST="xx"
	if [ "$NAS_CLASS" = "D" ]; then
		REMOVE_LIST="ft dc"
	fi
	for REMOVE in $REMOVE_LIST; do
		JOBLIST=`echo $JOBLIST | sed -e "s/$REMOVE//"`
	done
	export OMP_NUM_THREADS=$NAS_MAX_CPUS
	if [ "$NAS_BIND" = "yes" ]; then
		export OMP_PROC_BIND=true
	fi
	;;
*)
	die "Unrecognised NAS type $NAS_TYPE"
	;;
esac

if [ "$RUNBITS" = "" ]; then
	case `uname -m` in
		i?86)
			RUNBITS=32
			;;
		*)
			RUNBITS=64
			;;
	esac
fi
export PATH=$SHELLPACK_TOPLEVEL/nas/bin:$PATH

function lookup_mpicpus
{
	local NAS_JOB=$1

	JOB_MAX_CPUS=$NAS_MAX_CPUS
	for SPEC in $NAS_JOB_MAX_CPUS; do
		SPEC_JOB=`echo $SPEC | awk -F = '{print $1}'`
		SPEC_CPUS=`echo $SPEC | awk -F = '{print $2}'`

		if [ "$SPEC_JOB" = "$NAS_JOB" ]; then
			JOB_MAX_CPUS=$SPEC_CPUS
		fi
	done

	case $NAS_JOB in
	cg|ft|is|mg)
		MPICPUS=$(round_down_power_2 $JOB_MAX_CPUS)
		;;
	*)
		MPICPUS=$(round_down_nearest_square $JOB_MAX_CPUS)
		;;
	esac
}

# Generate suite.def
SUBVERSION=$VERSION
if [ ! -e $SHELLPACK_SOURCES/nas-$VERSION-installed/NPB$VERSION-$NAS_TYPE ]; then
	SUBVERSION=`echo $VERSION | sed -e 's/\.[0-9]*$//'`
fi
pushd $SHELLPACK_SOURCES/nas-$VERSION-installed/NPB$SUBVERSION-$NAS_TYPE > /dev/null || die "Failed to cd to nas-$VERSION-installed/NPB$SUBVERSION-$NAS_TYPE"
rm -rf bin
mkdir -p bin
echo -n > config/suite.def
for JOB in $JOBLIST; do
	lookup_mpicpus $JOB
	echo "$JOB      $NAS_CLASS $MPICPUS" >> config/suite.def
done

# Run the benchmark ##########################################################
export GCC_OPTIMISE=$NAS_OPTIMISE_FLAGS
if [ "$GCC_OPTIMISE" = "" ]; then
	export GCC_OPTIMISE=$DEFAULT_OPTIMISATION_LEVEL
fi
GENSPECSH="generate-nas.sh --bitness ${RUNBITS} $OPENMP $OPENMPI"
$GENSPECSH > config/make.def
cp config/make.def $LOGDIR_RESULTS || die "Failed to generate make.def"
ulimit -n 4096

make clean || die "Failed to clean"
make suite || die "Failed to build"

for JOB in $JOBLIST; do
	mmtests_activity $JOB.$NAS_CLASS
	###SHELLPACK iteration_begin $NAS_ITERATIONS
		if [ "$RUNBITS" = "32" -a ! -x ./bin/$JOB.$NAS_CLASS ]; then
			echo "Skipping $JOB.$NAS_CLASS"
			continue
		fi

		echo Executing $JOB.$NAS_CLASS iteration $ITERATION/$NAS_ITERATIONS
		monitor_pre_hook $LOGDIR_RESULTS $JOB.$NAS_CLASS
		case $NAS_TYPE in
		SER|OMP)
			eval $TIME_CMD -o $LOGDIR_RESULTS/time-$JOB.$NAS_CLASS.$ITERATION	\
				$NAS_WRAPPER ./bin/$JOB.$NAS_CLASS.x				\
				> $LOGDIR_RESULTS/$JOB.$NAS_CLASS.log.$ITERATION 2>&1
			;;
		MPI)
			lookup_mpicpus $JOB
			eval $TIME_CMD -o $LOGDIR_RESULTS/time-$JOB.$NAS_CLASS.$ITERATION	\
				$MPIRUN -np $MPICPUS $NAS_WRAPPER ./bin/$JOB.$NAS_CLASS.$MPICPUS \
				> $LOGDIR_RESULTS/$JOB.$NAS_CLASS.log.$ITERATION 2>&1
			;;
		esac

		if [ $? -ne 0 ]; then
			die "Failed to execute $JOB.$NAS_CLASS"
		fi
		monitor_post_hook $LOGDIR_RESULTS $JOB.$NAS_CLASS

		rm `find -name "ADC.view.dat*"` 2> /dev/null
	###SHELLPACK iteration_end $NAS_ITERATIONS
done

exit $SHELLPACK_SUCCESS
